{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-1-b28a23f23dd9>:9: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,\n"
     ]
    }
   ],
   "source": [
    "# 模块准备\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml.html import fromstring\n",
    "import time\n",
    "from random import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-3-dc413a9e1190>:7: DeprecationWarning: use options instead of chrome_options\n",
      "  browser = webdriver.Chrome(executable_path=r'C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)\n"
     ]
    }
   ],
   "source": [
    "# selenium指定路径下载文件\n",
    "from selenium import webdriver\n",
    "options = webdriver.ChromeOptions()\n",
    "out_path = r'E:\\数据挖掘\\期末项目\\数据文件'  # 是你想指定的路径\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': out_path}\n",
    "options.add_experimental_option('prefs', prefs)\n",
    "browser = webdriver.Chrome(executable_path=r'C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 打开CNKI。校外登陆"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 打开\n",
    "driver.get('https://cnki.net')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('/html/body/div[1]/div[1]/div/div/div/div[1]/div/div[2]/div[2]/div[1]/a')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "payload = {\"text\":\"15218561229\",\"password\":\"whxxhsnl0.\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//input[@name=\"Ecp_TextBoxUserName\"]')\n",
    "element.get_attribute('innerHTML')\n",
    "element.clear()\n",
    "element.send_keys(payload['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//input[@name=\"Ecp_TextBoxPwd\"]')\n",
    "element.get_attribute('innerHTML')\n",
    "element.clear()\n",
    "element.send_keys(payload['password'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//a[@class=\"login-alert-btn2\"]')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 检查登录\n",
    "element = driver.find_element_by_id('Ecp_loginShowName1')\n",
    "element.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  打开高级检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "高级检索\n"
     ]
    }
   ],
   "source": [
    "element=driver.find_element_by_id('highSearch')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 检查窗口\n",
    "* 只要是多个窗口，请检查窗口位置信息\n",
    "* 窗口信息根据窗口加载顺序按照列表存储，每一个窗口有唯一识别的窗口ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CDwindow-3DAA5D57995FAB6D6F18551D34067200',\n",
       " 'CDwindow-E1DB304B73A9BB490EB8C4EC79FA7C6A']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 所有窗口ID\n",
    "driver.window_handles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CDwindow-3DAA5D57995FAB6D6F18551D34067200'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 当前窗口\n",
    "driver.current_window_handle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-14-704fdf3805c1>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "# 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 点击学术期刊以及专业检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击学术期刊\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div[1]/div/ul[1]/li[1]/a')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 专业检索\n",
    "element = driver.find_element_by_name('majorSearch')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 选择期刊来源（SCI 、EI与CSSCI）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SCI \n",
    "element=driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[2]/input')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EI\n",
    "element=driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[3]/input')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CSSCI\n",
    "element=driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[5]/input')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置搜索query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置query\n",
    "query = 'SU=\"政治\" AND (TI=\"媒体\" OR TI =\"党\")'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输入检索要求\n",
    "element = driver.find_element_by_xpath('//textarea')\n",
    "element.clear()\n",
    "element.send_keys(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击检索\n",
    "element = driver.find_element_by_xpath('//input[@value=\"检索\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'共找到<em>5,265</em>条结果'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 检查数量\n",
    "element = driver.find_element_by_xpath('//span[@class=\"pagerTitleCell\"]')\n",
    "element.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 点击更换显示页数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击更换显示页数\n",
    "element = driver.find_element_by_xpath('//i[@class=\"icon icon-sort\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//div[@id=\"perPageDiv\"]//li[@data-val=\"50\"]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导出refworks文件（.txt）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "全选 = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/label/input')\n",
    "全选.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 5, 6, 7, 8]\n",
      "[9, 10, 11, 12, 13]\n"
     ]
    }
   ],
   "source": [
    "# 页数\n",
    "pages1 = list(range(0,9))\n",
    "pages2 =list(range(9,14))\n",
    "print(pages1)\n",
    "print(pages2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selectCheckbox (pages1):\n",
    "    for p in pages1:\n",
    "        print (p,end='\\t')\n",
    "        跳转 = driver.find_element_by_id('PageNext')\n",
    "        跳转.click()\n",
    "        time.sleep(10+5*random())\n",
    "        全选 = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/label/input')\n",
    "        全选.click()\n",
    "        time.sleep(5+5*random())  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\t1\t2\t3\t4\t5\t6\t7\t8\t"
     ]
    }
   ],
   "source": [
    "selectCheckbox (pages1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导出前500篇\n",
    "# 点击导出与分析\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/i')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击导出文献\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择Refworks\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/ul/li[8]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CDwindow-3DAA5D57995FAB6D6F18551D34067200',\n",
       " 'CDwindow-E1DB304B73A9BB490EB8C4EC79FA7C6A',\n",
       " 'CDwindow-D50A4ABEA388B718E554EAA3C61C8426']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 查看所有窗口\n",
    "driver.window_handles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-34-24f0cf647c1b>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "# 跳转到第三个页面\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击导出前500篇txt文件\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[1]/ul/li[3]/a')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击批量下载前500篇文章\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[1]/ul/li[2]/a')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-37-75aca382f043>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[3])\n"
     ]
    }
   ],
   "source": [
    "# 跳转到第四个页面\n",
    "driver.switch_to_window(driver.window_handles[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击批量下载\n",
    "element = driver.find_element_by_xpath('/html/body/div[1]/div/div[1]/button')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 后300篇"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-39-b24a4b09c9a8>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "# 跳转到第二个页面\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[2]/a[8]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 清除已选数据\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selectCheckbox (pages2):\n",
    "    for p in pages2:\n",
    "        print (p,end='\\t')\n",
    "        跳转 = driver.find_element_by_id('PageNext')\n",
    "        跳转.click()\n",
    "        time.sleep(10+10*random())\n",
    "        全选 = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/label/input')\n",
    "        全选.click()\n",
    "        time.sleep(5+10*random())  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\t10\t11\t12\t13\t"
     ]
    }
   ],
   "source": [
    "selectCheckbox (pages2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导出与分析\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/i')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导出文献\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择Refworks\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/ul[1]/li[2]/ul/li[1]/ul/li[8]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-47-24f0cf647c1b>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "# 跳转到第三个页面\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导出后300篇txt文件\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[1]/ul/li[3]/a')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 批量下载后300篇文章\n",
    "element = driver.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[1]/ul/li[2]/a')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-50-75aca382f043>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[3])\n"
     ]
    }
   ],
   "source": [
    "# 跳转到第四个页面\n",
    "driver.switch_to_window(driver.window_handles[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击批量下载\n",
    "element = driver.find_element_by_xpath('/html/body/div[1]/div/div[1]/button')\n",
    "element.get_attribute('innerHTML')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 抓取页面信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import requests\n",
    "import requests_html\n",
    "from requests_html import HTMLSession\n",
    "from lxml import etree\n",
    "from lxml.html import fromstring\n",
    "from urllib.parse import urljoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-53-b24a4b09c9a8>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "# 跳转到第二个页面\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 清除已选数据\n",
    "element =driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[2]/div[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOST = \"https://kns.cnki.net/\"\n",
    "html = driver.page_source\n",
    "soup = etree.HTML(html)\n",
    "tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "headers = {\n",
    "    \"Cookie\":'Ecp_ClientId=3200615112602316607; cnkiUserKey=03369524-5653-0508-82c6-2fdbcdb21fe4; RsPerPage=20; _pk_ref=%5B%22%22%2C%22%22%2C1607047617%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; Ecp_ClientIp=202.116.81.140; UM_distinctid=178689abc586d3-0e68ae30691535-5771133-1fa400-178689abc597f5; Ecp_loginuserjf=15014134753; Ecp_session=1; ASP.NET_SessionId=gt3afeursglwft12mavvuvzf; SID_kns8=123111; SID_kns_new=kns123106; CurrSortFieldType=desc; SID_kcms=124103; SID_kxreader_new=011121; Hm_lvt_6e967eb120601ea41b9d312166416aa6=1621474773; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%7D%2C%22%24device_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%7D; Hm_lvt_ba7af201fc75865e9846f701ccb53e6b=1621474773; SID_kns=025123113; SID_klogin=125144; Hm_lpvt_6e967eb120601ea41b9d312166416aa6=1621475961; Hm_lpvt_ba7af201fc75865e9846f701ccb53e6b=1621475961; Ecp_loginuserbk=GDZSDX; knsLeftGroupSelectItem=null5%3B9%3B; CurrSortField=%e8%a2%ab%e5%bc%95%2f(%e8%a2%ab%e5%bc%95%e9%a2%91%e6%ac%a1%2c%27integer%27); _pk_ref=%5B%22%22%2C%22%22%2C1622444666%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; _pk_ses=*; _pk_id=90adc29b-dc70-48ff-8dcc-60e442d5a980.1619271339.5.1622444668.1622444666.; Ecp_LoginStuts={\"IsAutoLogin\":false,\"UserName\":\"GZ0513\",\"ShowName\":\"%e4%b8%ad%e5%b1%b1%e5%a4%a7%e5%ad%a6%e5%8d%97%e6%96%b9%e5%ad%a6%e9%99%a2\",\"UserType\":\"bk\",\"BUserName\":\"\",\"BShowName\":\"\",\"BUserType\":\"\",\"r\":\"4SVIml\"}; LID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!; c_m_LinID=LinID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!&ot=05/31/2021 15:24:33; c_m_expire=2021-05-31 15:24:33',\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'\n",
    "}\n",
    "def get_data(href):   #一个封装好的完善详细页链接的函数\n",
    "    DbCode = re.findall(\"DbCode=(.*?)&\",href)[0]\n",
    "    dbname = re.findall(\"dbname=(.*?)&\",href)[0]\n",
    "    filename = re.findall(\"filename=(.*?)&\",href)[0]\n",
    "    href = f\"https://kns.cnki.net/kcms/detail/detail.aspx?dbcode={DbCode}&dbname={dbname}&filename={filename}\"\n",
    "    return href\n",
    "\n",
    "def download(href):   #一个封装好的获取PDF下载链接的函数\n",
    "    page_source = requests.get(url=href,headers=headers).text\n",
    "    page_source_soup = etree.HTML(page_source)\n",
    "    download_url = page_source_soup.xpath('//li[@class=\"btn-dlpdf\"]/a/@href')\n",
    "    if download_url:\n",
    "        return urljoin(HOST,download_url[0])\n",
    "    else:\n",
    "        return \"无下载链接\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pages =18   \n",
    "\n",
    "title_list = []\n",
    "author_list = []\n",
    "href_list = []\n",
    "download_url_list = []\n",
    "\n",
    "\n",
    "def 数据():\n",
    "    now_page_count = 1\n",
    "    while now_page_count < num_pages:\n",
    "        html = driver.page_source\n",
    "        soup = etree.HTML(html)\n",
    "        tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "        for tr in tr_list:\n",
    "            title = \"\".join(tr.xpath('td[@class=\"name\"]//text()')).strip().replace(\"\\n\",\"\").replace(\"                              网络首发\",\"\")\n",
    "            author = \",\".join(tr.xpath('td[@class=\"author\"]/a//text()')).strip().replace(\"\\n\",\"\")\n",
    "            href = tr.xpath('td[@class=\"name\"]/a/@href')[0]\n",
    "            href = get_data(href)\n",
    "            download_url = download(href)\n",
    "            title_list.append(title)\n",
    "            author_list.append(author)\n",
    "            href_list.append(href)\n",
    "            download_url_list.append(download_url)\n",
    "        now_page_count = now_page_count + 1\n",
    "        driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "        time.sleep(0.5)\n",
    "    data = {\"标题\":title_list,\"作者\":author_list,\"详细页链接\":href_list,\"PDF下载链接\":download_url_list}\n",
    "    data = pd.DataFrame(data)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>作者</th>\n",
       "      <th>详细页链接</th>\n",
       "      <th>PDF下载链接</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>高校思想政治教学模式研究——评《新媒体时代高校思想政治教育的解构与重塑》</td>\n",
       "      <td>周岚峰</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>大学生思想政治教育创新研究——评《新媒体视域下大学生思想政治教育研究》</td>\n",
       "      <td>谷秀娟</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>新时代党的政治生态建设研究</td>\n",
       "      <td>周叶婷</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>新时代加强党的政治建设的逻辑意蕴</td>\n",
       "      <td>上官酒瑞</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>时政微视频:主流媒体时政报道的创新探索</td>\n",
       "      <td>杨明珠</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>新媒体时代大学生政治价值观认同研究</td>\n",
       "      <td>李庆华,贾凯丽</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>美国两党政治走向及对特朗普外交的影响</td>\n",
       "      <td>刁大明</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>社交媒体时代西方政党的调适性变革及其新困境</td>\n",
       "      <td>陈文胜</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>848</th>\n",
       "      <td>中国特色社会主义是改革开放以来党的全部理论和实践的主题</td>\n",
       "      <td>黄百炼</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>849</th>\n",
       "      <td>坦桑尼亚革命党的角色转变及其面临的挑战</td>\n",
       "      <td>李鹏涛</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>850 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       标题       作者  \\\n",
       "0    高校思想政治教学模式研究——评《新媒体时代高校思想政治教育的解构与重塑》      周岚峰   \n",
       "1     大学生思想政治教育创新研究——评《新媒体视域下大学生思想政治教育研究》      谷秀娟   \n",
       "2                           新时代党的政治生态建设研究      周叶婷   \n",
       "3                        新时代加强党的政治建设的逻辑意蕴     上官酒瑞   \n",
       "4                     时政微视频:主流媒体时政报道的创新探索      杨明珠   \n",
       "..                                    ...      ...   \n",
       "845                     新媒体时代大学生政治价值观认同研究  李庆华,贾凯丽   \n",
       "846                    美国两党政治走向及对特朗普外交的影响      刁大明   \n",
       "847                 社交媒体时代西方政党的调适性变革及其新困境      陈文胜   \n",
       "848           中国特色社会主义是改革开放以来党的全部理论和实践的主题      黄百炼   \n",
       "849                   坦桑尼亚革命党的角色转变及其面临的挑战      李鹏涛   \n",
       "\n",
       "                                                 详细页链接  \\\n",
       "0    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "1    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "2    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "3    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "4    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "..                                                 ...   \n",
       "845  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "846  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "847  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "848  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "849  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "\n",
       "                                               PDF下载链接  \n",
       "0    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "1    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "2    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "3    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "4    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "..                                                 ...  \n",
       "845  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "846  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "847  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "848  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "849  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "\n",
       "[850 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#输出数据做成表格表格\n",
    "data = 数据()\n",
    "display(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#存入Excel\n",
    "data.to_excel('知网数据.xlsx',sheet_name=\"＞800篇\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 下载pdf档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-59-7320f41f93bd>:20: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome(executable_path=r'C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)\n"
     ]
    }
   ],
   "source": [
    "# 准备工作\n",
    "#caps=dict()\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\n",
    "\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "#opts.add_argument('--headless') #浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败\n",
    "# opts.binary_location = \"C:\\portable\\PortableApps\\IronPortable\\App\\Iron\\chrome.exe\"\n",
    "# opts.binary_location = \"C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe\" #\"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "\n",
    "\n",
    "options = webdriver.ChromeOptions()\n",
    "out_path = r'E:\\数据挖掘\\期末项目\\数据文件'  # 是你想指定的路径\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': out_path}\n",
    "options.add_experimental_option('prefs', prefs)\n",
    "driver = webdriver.Chrome(executable_path=r'C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)\n",
    "#driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调用API解决验证码问题\n",
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]\n",
    "    return \"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "党的政治建设的核心要义、价值意蕴和实践路径----已下载\n",
      "科学与“党义”:国民历编订的争论及其意蕴----已下载\n",
      "新媒体环境下高校网络隐性思想政治教育策略研究----已下载\n",
      "论新时代中国多党合作制度话语体系建设----已下载\n",
      "新媒体、民众对基层政府的人际信任与民族和谐感知——来自X地区的证据----已下载\n",
      "思想从严:全面从严治党纵深发展的意识形态内生逻辑----已下载\n",
      "把党建设得更加坚强有力----已下载\n",
      "以政治建设统领全面从严治党----已下载\n",
      "全面从严治党必须加强政德建设----已下载\n",
      "全媒体语境下国家话语体系建构的双重转向----已下载\n",
      "论新时代党的政治建设的内涵与结构----已下载\n",
      "新媒体环境下政党形象建构的适应性分析----已下载\n",
      "新媒体环境下网络动员双重机制探究----已下载\n",
      "改革开放40年党的政治文化演进逻辑与创新研究----已下载\n",
      "思想建党和制度治党结合的内在逻辑、制约因素与实现路径----已下载\n",
      "党内法规视域下的党领导立法----已下载\n",
      "全面从严治党实践中“圈子文化”乱象治理探析----已下载\n",
      "“党的建设”二级学科建设的思路----已下载\n",
      "政治建设、组织力与党的建设质量——新时代党的建设三大新概念新要求----已下载\n",
      "新媒体与思想政治教育的有机结合——评《新媒体与大学生思想政治教育研究》----已下载\n",
      "法治中国的历史演进——兼论依规治党的历史方位----已下载\n",
      "全面从严治党与完善和落实民主集中制----已下载\n",
      "新时代中国共产党的伟大使命与党的建设新要求----已下载\n",
      "关于“思想道德修养与法律基础”课教学贯彻党的十九大精神的思考----已下载\n",
      "党的新闻发布工作与社会治理现代化初探----已下载\n",
      "杜特尔特治下菲律宾民力党的崛起及愿景----已下载\n",
      "在全面从严治党中探索执政规律----已下载\n",
      "高层次人才建设更要坚持“党管人才”原则----已下载\n",
      "习近平在中共中央政治局第六次集体学习时强调  把党的政治建设作为党的根本性建设  为党不断从胜利走向胜利提供重要保证----已下载\n"
     ]
    }
   ],
   "source": [
    "# 读取上面已保存在本地的xlsx文件【利用里面的pdf下载链接】\n",
    "data = pd.read_excel(\"知网数据.xlsx\") \n",
    "base = \"https://kns.cnki.net/\"\n",
    "# 开始与结束的链接序号 count end_count\n",
    "count = 421\n",
    "end_count = 450 \n",
    "# 读取title、pdf链接\n",
    "title_list = data[\"标题\"].to_list()[count:end_count]\n",
    "download_url_list = data[\"PDF下载链接\"].to_list()[count:end_count]\n",
    "list_all_url_list = data[\"详细页链接\"].to_list()[count:end_count]\n",
    "for title,download_url,list_all_url in zip(title_list,download_url_list,list_all_url_list):\n",
    "    try:\n",
    "        # 进入pdf的下载链接\n",
    "        driver.get(download_url)\n",
    "        time.sleep(5)\n",
    "        # 出现验证码的情况\n",
    "        if driver.find_element_by_xpath('//*[@id=\"vImg\"]'):\n",
    "            time.sleep(5)\n",
    "            ele = driver.find_element_by_id('vImg')\n",
    "            ele.screenshot(r'E:\\数据挖掘\\期末项目\\数据文件\\验证码.png')\n",
    "            #截取全屏后定位截图里验证码的位置，并重新裁剪保存\n",
    "            # api识别部分\n",
    "            img_path = r\"E:\\数据挖掘\\期末项目\\数据文件\\验证码.png\"\n",
    "            result = base64_api(uname='gyu', pwd='20001026',img=img_path,typeid=3)\n",
    "            # 输出识别出的验证码\n",
    "            print(\"验证码识别：\",result) \n",
    "            # 将验证码填入\n",
    "            driver.find_element_by_id('vcode').send_keys(result)\n",
    "            time.sleep(5)\n",
    "            # 点击验证码的提交，然后开始下载\n",
    "            driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()\n",
    "    except:\n",
    "        print(f'{title}----已下载')\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
